In [3]:
import numpy as np
import pandas as pd
import matplotlib.image as mpimg
import matplotlib.pyplot as plt
from tqdm import tqdm
import os
from os.path import isfile, join
import random
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras.preprocessing.image import load_img
from tensorflow.keras import layers
from tensorflow.keras import metrics
In [5]:
DIR = "/Users/sandeepkumar/Downloads/archive/lgg-mri-segmentation/kaggle_3m"
INPUT_CHANNELS = 3
TARGET_CHANNELS = 1
SIZE = 256
BATCH_SIZE = 32
In [7]:
mri_images_with_tumer = []
mri_images_without_tumer = []
mask_images_with_tumer = []
mask_images_without_tumer = []
patients = os.listdir(DIR)
for patient in tqdm(patients):
if isfile(join(DIR, patient)) == False:
images = os.listdir(join(DIR, patient))
mask_images = list(filter(lambda x: x.find('mask') != -1, images))
mri_images = list(filter(lambda x: x.find('mask') == -1, images))
for mask_image in mask_images:
mask = np.asarray(load_img(
join(DIR, patient, mask_image),
target_size=(SIZE, SIZE),
color_mode="grayscale"))
if np.amax(mask) != 0:
mri_images_with_tumer.append(join(patient, mask_image.replace('_mask', '')))
mask_images_with_tumer.append(join(patient, mask_image))
else:
mri_images_without_tumer.append(join(patient, mask_image.replace('_mask', '')))
mask_images_without_tumer.append(join(patient, mask_image))
100%|█████████████████████████████████████████| 114/114 [00:01<00:00, 70.95it/s]
In [9]:
random.Random(1337).shuffle(mri_images_with_tumer)
random.Random(1337).shuffle(mask_images_with_tumer)
random.Random(1337).shuffle(mri_images_without_tumer)
random.Random(1337).shuffle(mask_images_without_tumer)
print("Total MRI images: ", len(mri_images_with_tumer) + len(mri_images_without_tumer))
print("Total mask images: ", len(mask_images_with_tumer) + len(mask_images_without_tumer))
print("Total images with tumer: ", len(mri_images_with_tumer))
print("Total images without tumer: ", len(mri_images_without_tumer))
labels = ['With Tumer', 'Without Tumer']
count = [len(mri_images_with_tumer), len(mri_images_without_tumer)]
fig, ax = plt.subplots()
ax.pie(count, labels=labels, autopct='%1.1f%%', shadow=True, startangle=90)
ax.axis('equal') # Equal aspect ratio ensures that pie is drawn as a circle.
plt.show()
Total MRI images: 3929 Total mask images: 3929 Total images with tumer: 1373 Total images without tumer: 2556
In [11]:
mri_images_with_tumer = np.array(mri_images_with_tumer)
mri_images_without_tumer = np.array(mri_images_without_tumer)
mask_images_with_tumer = np.array(mask_images_with_tumer)
mask_images_without_tumer = np.array(mask_images_without_tumer)
In [13]:
with_tumer_val_images = 300
without_tumer_val_images = 600
with_tumer_test_images = 10
without_tumer_test_images = 5
train_images = np.concatenate(
(mri_images_with_tumer[:-with_tumer_val_images-with_tumer_test_images],
mri_images_without_tumer[:-without_tumer_val_images-without_tumer_test_images]),
axis = 0)
val_images = np.concatenate(
(mri_images_with_tumer[-with_tumer_val_images-with_tumer_test_images:-with_tumer_test_images],
mri_images_without_tumer[-without_tumer_val_images-without_tumer_test_images:-without_tumer_test_images]),
axis = 0)
test_images = np.concatenate(
(mri_images_with_tumer[-with_tumer_test_images:],
mri_images_without_tumer[-without_tumer_test_images:]),
axis = 0)
train_targets = np.concatenate(
(mask_images_with_tumer[:-with_tumer_val_images-with_tumer_test_images],
mask_images_without_tumer[:-without_tumer_val_images-without_tumer_test_images]),
axis = 0)
val_targets = np.concatenate(
(mask_images_with_tumer[-with_tumer_val_images-with_tumer_test_images:-with_tumer_test_images],
mask_images_without_tumer[-without_tumer_val_images-without_tumer_test_images:-without_tumer_test_images]),
axis = 0)
test_targets = np.concatenate(
(mask_images_with_tumer[-with_tumer_test_images:],
mask_images_without_tumer[-without_tumer_test_images:]),
axis = 0)
print("train_images: ", train_images.shape)
print("train_targets: ", train_targets.shape)
print("val_images: ", val_images.shape)
print("val_targets: ", val_targets.shape)
print("test_images: ", test_images.shape)
print("test_targets: ", test_targets.shape)
train_images: (3014,) train_targets: (3014,) val_images: (900,) val_targets: (900,) test_images: (15,) test_targets: (15,)
In [15]:
random.Random(37).shuffle(train_images)
random.Random(37).shuffle(train_targets)
random.Random(37).shuffle(val_images)
random.Random(37).shuffle(val_targets)
In [17]:
train_df = pd.DataFrame(data={'mris': train_images, 'masks': train_targets})
val_df = pd.DataFrame(data={'mris': val_images, 'masks': val_targets})
test_df = pd.DataFrame(data={'mris': test_images, 'masks': test_targets})
In [19]:
def adjust_data(img,mask):
img = img / 255
# imt = 1 - img
mask = mask / 255
mask[mask > 0.5] = 1
mask[mask <= 0.5] = 0
return (img, mask)
class BrainMRIs(keras.utils.Sequence):
def __init__(self, batch_size, img_size, input_img_paths, target_img_paths, directory):
self.batch_size = batch_size
self.img_size = img_size
self.input_img_paths = input_img_paths
self.target_img_paths = target_img_paths
self.directory = directory
def __len__(self):
return len(self.target_img_paths) // self.batch_size
def __getitem__(self, idx):
i = idx * self.batch_size
batch_input_img_paths = self.input_img_paths[i : i + self.batch_size]
batch_target_img_paths = self.target_img_paths[i : i + self.batch_size]
x = np.zeros((self.batch_size, self.img_size, self.img_size, 3), dtype="float32")
for j, path in enumerate(batch_input_img_paths):
img = load_img(join(self.directory, path), target_size=(self.img_size, self.img_size))
x[j] = img
y = np.zeros((self.batch_size, self.img_size, self.img_size, 1), dtype="float32")
for j, path in enumerate(batch_target_img_paths):
img = load_img(join(self.directory, path), target_size=(self.img_size, self.img_size), color_mode="grayscale")
y[j] = np.expand_dims(img, 2)
return adjust_data(x, y)
In [21]:
train_gen = BrainMRIs(BATCH_SIZE, SIZE, train_images, train_targets, DIR)
val_gen = BrainMRIs(BATCH_SIZE, SIZE, val_images, val_targets, DIR)
test_gen = BrainMRIs(BATCH_SIZE, SIZE, test_images, test_targets, DIR)
In [23]:
smooth=100
def dice_coef(y_true, y_pred):
y_truef = keras.backend.flatten(y_true)
y_predf = keras.backend.flatten(y_pred)
And = keras.backend.sum(y_truef*y_predf)
return((2* And + smooth) / (keras.backend.sum(y_truef) + keras.backend.sum(y_predf) + smooth))
def dice_coef_loss(y_true, y_pred):
return -dice_coef(y_true, y_pred)
def iou(y_true, y_pred):
intersection = keras.backend.sum(y_true * y_pred)
sum_ = keras.backend.sum(y_true + y_pred)
jac = (intersection + smooth) / (sum_ - intersection + smooth)
return jac
def jac_distance(y_true, y_pred):
y_truef = keras.backend.flatten(y_true)
y_predf = keras.backend.flatten(y_pred)
return - iou(y_true, y_pred)
In [25]:
from keras.losses import binary_crossentropy
from tensorflow.keras import backend as K
epsilon = 1e-5
smooth = 1
def tversky(y_true, y_pred):
y_true_pos = K.flatten(y_true)
y_pred_pos = K.flatten(y_pred)
true_pos = K.sum(y_true_pos * y_pred_pos)
false_neg = K.sum(y_true_pos * (1-y_pred_pos))
false_pos = K.sum((1-y_true_pos)*y_pred_pos)
alpha = 0.7
return (true_pos + smooth)/(true_pos + alpha*false_neg + (1-alpha)*false_pos + smooth)
def focal_tversky(y_true,y_pred):
y_true = tf.cast(y_true, tf.float32)
y_pred = tf.cast(y_pred, tf.float32)
pt_1 = tversky(y_true, y_pred)
gamma = 0.75
return K.pow((1-pt_1), gamma)
def tversky_loss(y_true, y_pred):
return 1 - tversky(y_true,y_pred)
In [27]:
keras.backend.clear_session()
In [29]:
inputs = layers.Input((SIZE, SIZE, INPUT_CHANNELS))
c1 = layers.Conv2D(32, kernel_size=(3, 3), activation='relu', padding='same', name='conv1_1')(inputs)
# c1 = layers.Dropout(0.2)(c1)
c1 = layers.Conv2D(32, kernel_size=(3, 3), padding='same', name="conv1_2")(c1)
c1 = layers.BatchNormalization(axis=3)(c1)
c1 = layers.Activation('relu')(c1)
input_1 = layers.MaxPooling2D((2, 2))(c1)
c2 = layers.Conv2D(64, kernel_size=(3, 3), activation='relu', padding='same', name='conv2_1')(input_1)
# c2 = layers.Dropout(0.2)(c2)
c2 = layers.Conv2D(64, kernel_size=(3, 3), padding='same', name="conv2_2")(c2)
c2 = layers.BatchNormalization(axis=3)(c2)
c2 = layers.Activation('relu')(c2)
input_2 = layers.MaxPooling2D((2, 2))(c2)
c3 = layers.Conv2D(128, kernel_size=(3, 3), activation='relu', padding='same', name='conv3_1')(input_2)
# c3 = layers.Dropout(0.2)(c3)
c3 = layers.Conv2D(128, kernel_size=(3, 3), padding='same', name="conv3_2")(c3)
c3 = layers.BatchNormalization(axis=3)(c3)
c3 = layers.Activation('relu')(c3)
input_3 = layers.MaxPooling2D((2, 2))(c3)
c4 = layers.Conv2D(256, kernel_size=(3, 3), activation='relu', padding='same', name='conv4_1')(input_3)
# c4 = layers.Dropout(0.2)(c4)
c4 = layers.Conv2D(256, kernel_size=(3, 3), padding='same', name="conv4_2")(c4)
c4 = layers.BatchNormalization(axis=3)(c4)
c4 = layers.Activation('relu')(c4)
input_4 = layers.MaxPooling2D((2, 2))(c4)
c5 = layers.Conv2D(512, kernel_size=(3, 3), activation='relu', padding='same', name='conv5_1')(input_4)
# c5 = layers.Dropout(0.2)(c5)
c5 = layers.Conv2D(512, kernel_size=(3, 3), padding='same', name="conv5_2")(c5)
c5 = layers.BatchNormalization(axis=3)(c5)
c5 = layers.Activation('relu')(c5)
u6 = layers.Conv2DTranspose(256, kernel_size=(2, 2), strides=(2, 2), activation='relu', padding='same', name='conv6_1')(c5)
u6 = layers.concatenate([u6, c4])
c6 = layers.Conv2D(256, kernel_size=(3, 3), activation='relu', padding='same', name='conv6_2')(u6)
# c6 = layers.Dropout(0.2)(c6)
c6 = layers.Conv2D(256, kernel_size=(3, 3), padding='same', name="conv6_3")(c6)
c6 = layers.BatchNormalization(axis=3)(c6)
c6 = layers.Activation('relu')(c6)
u7 = layers.Conv2DTranspose(128, kernel_size=(2, 2), strides=(2, 2), activation='relu', padding='same', name='conv7_1')(c6)
u7 = layers.concatenate([u7, c3])
c7 = layers.Conv2D(128, kernel_size=(3, 3), activation='relu', padding='same', name='conv7_2')(u7)
# c7 = layers.Dropout(0.2)(c7)
c7 = layers.Conv2D(128, kernel_size=(3, 3), padding='same', name="conv7_3")(c7)
c7 = layers.BatchNormalization(axis=3)(c7)
c7 = layers.Activation('relu')(c7)
u8 = layers.Conv2DTranspose(64, kernel_size=(2, 2), strides=(2, 2), activation='relu', padding='same', name='conv8_1')(c7)
u8 = layers.concatenate([u8, c2])
c8 = layers.Conv2D(64, kernel_size=(3, 3), activation='relu', padding='same', name='conv8_2')(u8)
# c8 = layers.Dropout(0.2)(c8)
c8 = layers.Conv2D(64, kernel_size=(3, 3), padding='same', name="conv8_3")(c8)
c8 = layers.BatchNormalization(axis=3)(c8)
c8 = layers.Activation('relu')(c8)
u9 = layers.Conv2DTranspose(32, kernel_size=(2, 2), strides=(2, 2), activation='relu', padding='same', name='conv9_1')(c8)
u9 = layers.concatenate([u9, c1])
c9 = layers.Conv2D(32, kernel_size=(3, 3), activation='relu', padding='same', name='conv9_2')(u9)
# c9 = layers.Dropout(0.2)(c9)
c9 = layers.Conv2D(32, kernel_size=(3, 3), padding='same', name="conv9_3")(c9)
c9 = layers.BatchNormalization(axis=3)(c9)
c9 = layers.Activation('relu')(c9)
outputs = layers.Conv2D(1, kernel_size=(1, 1), activation='sigmoid', padding='same', name="output")(c9)
model = keras.Model(inputs=[inputs], outputs=[outputs])
model.summary()
Model: "functional"
┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┓ ┃ Layer (type) ┃ Output Shape ┃ Param # ┃ Connected to ┃ ┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━┩ │ input_layer │ (None, 256, 256, │ 0 │ - │ │ (InputLayer) │ 3) │ │ │ ├─────────────────────┼───────────────────┼────────────┼───────────────────┤ │ conv1_1 (Conv2D) │ (None, 256, 256, │ 896 │ input_layer[0][0] │ │ │ 32) │ │ │ ├─────────────────────┼───────────────────┼────────────┼───────────────────┤ │ conv1_2 (Conv2D) │ (None, 256, 256, │ 9,248 │ conv1_1[0][0] │ │ │ 32) │ │ │ ├─────────────────────┼───────────────────┼────────────┼───────────────────┤ │ batch_normalization │ (None, 256, 256, │ 128 │ conv1_2[0][0] │ │ (BatchNormalizatio… │ 32) │ │ │ ├─────────────────────┼───────────────────┼────────────┼───────────────────┤ │ activation │ (None, 256, 256, │ 0 │ batch_normalizat… │ │ (Activation) │ 32) │ │ │ ├─────────────────────┼───────────────────┼────────────┼───────────────────┤ │ max_pooling2d │ (None, 128, 128, │ 0 │ activation[0][0] │ │ (MaxPooling2D) │ 32) │ │ │ ├─────────────────────┼───────────────────┼────────────┼───────────────────┤ │ conv2_1 (Conv2D) │ (None, 128, 128, │ 18,496 │ max_pooling2d[0]… │ │ │ 64) │ │ │ ├─────────────────────┼───────────────────┼────────────┼───────────────────┤ │ conv2_2 (Conv2D) │ (None, 128, 128, │ 36,928 │ conv2_1[0][0] │ │ │ 64) │ │ │ ├─────────────────────┼───────────────────┼────────────┼───────────────────┤ │ batch_normalizatio… │ (None, 128, 128, │ 256 │ conv2_2[0][0] │ │ (BatchNormalizatio… │ 64) │ │ │ ├─────────────────────┼───────────────────┼────────────┼───────────────────┤ │ activation_1 │ (None, 128, 128, │ 0 │ batch_normalizat… │ │ (Activation) │ 64) │ │ │ ├─────────────────────┼───────────────────┼────────────┼───────────────────┤ │ max_pooling2d_1 │ (None, 64, 64, │ 0 │ activation_1[0][… │ │ (MaxPooling2D) │ 64) │ │ │ ├─────────────────────┼───────────────────┼────────────┼───────────────────┤ │ conv3_1 (Conv2D) │ (None, 64, 64, │ 73,856 │ max_pooling2d_1[… │ │ │ 128) │ │ │ ├─────────────────────┼───────────────────┼────────────┼───────────────────┤ │ conv3_2 (Conv2D) │ (None, 64, 64, │ 147,584 │ conv3_1[0][0] │ │ │ 128) │ │ │ ├─────────────────────┼───────────────────┼────────────┼───────────────────┤ │ batch_normalizatio… │ (None, 64, 64, │ 512 │ conv3_2[0][0] │ │ (BatchNormalizatio… │ 128) │ │ │ ├─────────────────────┼───────────────────┼────────────┼───────────────────┤ │ activation_2 │ (None, 64, 64, │ 0 │ batch_normalizat… │ │ (Activation) │ 128) │ │ │ ├─────────────────────┼───────────────────┼────────────┼───────────────────┤ │ max_pooling2d_2 │ (None, 32, 32, │ 0 │ activation_2[0][… │ │ (MaxPooling2D) │ 128) │ │ │ ├─────────────────────┼───────────────────┼────────────┼───────────────────┤ │ conv4_1 (Conv2D) │ (None, 32, 32, │ 295,168 │ max_pooling2d_2[… │ │ │ 256) │ │ │ ├─────────────────────┼───────────────────┼────────────┼───────────────────┤ │ conv4_2 (Conv2D) │ (None, 32, 32, │ 590,080 │ conv4_1[0][0] │ │ │ 256) │ │ │ ├─────────────────────┼───────────────────┼────────────┼───────────────────┤ │ batch_normalizatio… │ (None, 32, 32, │ 1,024 │ conv4_2[0][0] │ │ (BatchNormalizatio… │ 256) │ │ │ ├─────────────────────┼───────────────────┼────────────┼───────────────────┤ │ activation_3 │ (None, 32, 32, │ 0 │ batch_normalizat… │ │ (Activation) │ 256) │ │ │ ├─────────────────────┼───────────────────┼────────────┼───────────────────┤ │ max_pooling2d_3 │ (None, 16, 16, │ 0 │ activation_3[0][… │ │ (MaxPooling2D) │ 256) │ │ │ ├─────────────────────┼───────────────────┼────────────┼───────────────────┤ │ conv5_1 (Conv2D) │ (None, 16, 16, │ 1,180,160 │ max_pooling2d_3[… │ │ │ 512) │ │ │ ├─────────────────────┼───────────────────┼────────────┼───────────────────┤ │ conv5_2 (Conv2D) │ (None, 16, 16, │ 2,359,808 │ conv5_1[0][0] │ │ │ 512) │ │ │ ├─────────────────────┼───────────────────┼────────────┼───────────────────┤ │ batch_normalizatio… │ (None, 16, 16, │ 2,048 │ conv5_2[0][0] │ │ (BatchNormalizatio… │ 512) │ │ │ ├─────────────────────┼───────────────────┼────────────┼───────────────────┤ │ activation_4 │ (None, 16, 16, │ 0 │ batch_normalizat… │ │ (Activation) │ 512) │ │ │ ├─────────────────────┼───────────────────┼────────────┼───────────────────┤ │ conv6_1 │ (None, 32, 32, │ 524,544 │ activation_4[0][… │ │ (Conv2DTranspose) │ 256) │ │ │ ├─────────────────────┼───────────────────┼────────────┼───────────────────┤ │ concatenate │ (None, 32, 32, │ 0 │ conv6_1[0][0], │ │ (Concatenate) │ 512) │ │ activation_3[0][… │ ├─────────────────────┼───────────────────┼────────────┼───────────────────┤ │ conv6_2 (Conv2D) │ (None, 32, 32, │ 1,179,904 │ concatenate[0][0] │ │ │ 256) │ │ │ ├─────────────────────┼───────────────────┼────────────┼───────────────────┤ │ conv6_3 (Conv2D) │ (None, 32, 32, │ 590,080 │ conv6_2[0][0] │ │ │ 256) │ │ │ ├─────────────────────┼───────────────────┼────────────┼───────────────────┤ │ batch_normalizatio… │ (None, 32, 32, │ 1,024 │ conv6_3[0][0] │ │ (BatchNormalizatio… │ 256) │ │ │ ├─────────────────────┼───────────────────┼────────────┼───────────────────┤ │ activation_5 │ (None, 32, 32, │ 0 │ batch_normalizat… │ │ (Activation) │ 256) │ │ │ ├─────────────────────┼───────────────────┼────────────┼───────────────────┤ │ conv7_1 │ (None, 64, 64, │ 131,200 │ activation_5[0][… │ │ (Conv2DTranspose) │ 128) │ │ │ ├─────────────────────┼───────────────────┼────────────┼───────────────────┤ │ concatenate_1 │ (None, 64, 64, │ 0 │ conv7_1[0][0], │ │ (Concatenate) │ 256) │ │ activation_2[0][… │ ├─────────────────────┼───────────────────┼────────────┼───────────────────┤ │ conv7_2 (Conv2D) │ (None, 64, 64, │ 295,040 │ concatenate_1[0]… │ │ │ 128) │ │ │ ├─────────────────────┼───────────────────┼────────────┼───────────────────┤ │ conv7_3 (Conv2D) │ (None, 64, 64, │ 147,584 │ conv7_2[0][0] │ │ │ 128) │ │ │ ├─────────────────────┼───────────────────┼────────────┼───────────────────┤ │ batch_normalizatio… │ (None, 64, 64, │ 512 │ conv7_3[0][0] │ │ (BatchNormalizatio… │ 128) │ │ │ ├─────────────────────┼───────────────────┼────────────┼───────────────────┤ │ activation_6 │ (None, 64, 64, │ 0 │ batch_normalizat… │ │ (Activation) │ 128) │ │ │ ├─────────────────────┼───────────────────┼────────────┼───────────────────┤ │ conv8_1 │ (None, 128, 128, │ 32,832 │ activation_6[0][… │ │ (Conv2DTranspose) │ 64) │ │ │ ├─────────────────────┼───────────────────┼────────────┼───────────────────┤ │ concatenate_2 │ (None, 128, 128, │ 0 │ conv8_1[0][0], │ │ (Concatenate) │ 128) │ │ activation_1[0][… │ ├─────────────────────┼───────────────────┼────────────┼───────────────────┤ │ conv8_2 (Conv2D) │ (None, 128, 128, │ 73,792 │ concatenate_2[0]… │ │ │ 64) │ │ │ ├─────────────────────┼───────────────────┼────────────┼───────────────────┤ │ conv8_3 (Conv2D) │ (None, 128, 128, │ 36,928 │ conv8_2[0][0] │ │ │ 64) │ │ │ ├─────────────────────┼───────────────────┼────────────┼───────────────────┤ │ batch_normalizatio… │ (None, 128, 128, │ 256 │ conv8_3[0][0] │ │ (BatchNormalizatio… │ 64) │ │ │ ├─────────────────────┼───────────────────┼────────────┼───────────────────┤ │ activation_7 │ (None, 128, 128, │ 0 │ batch_normalizat… │ │ (Activation) │ 64) │ │ │ ├─────────────────────┼───────────────────┼────────────┼───────────────────┤ │ conv9_1 │ (None, 256, 256, │ 8,224 │ activation_7[0][… │ │ (Conv2DTranspose) │ 32) │ │ │ ├─────────────────────┼───────────────────┼────────────┼───────────────────┤ │ concatenate_3 │ (None, 256, 256, │ 0 │ conv9_1[0][0], │ │ (Concatenate) │ 64) │ │ activation[0][0] │ ├─────────────────────┼───────────────────┼────────────┼───────────────────┤ │ conv9_2 (Conv2D) │ (None, 256, 256, │ 18,464 │ concatenate_3[0]… │ │ │ 32) │ │ │ ├─────────────────────┼───────────────────┼────────────┼───────────────────┤ │ conv9_3 (Conv2D) │ (None, 256, 256, │ 9,248 │ conv9_2[0][0] │ │ │ 32) │ │ │ ├─────────────────────┼───────────────────┼────────────┼───────────────────┤ │ batch_normalizatio… │ (None, 256, 256, │ 128 │ conv9_3[0][0] │ │ (BatchNormalizatio… │ 32) │ │ │ ├─────────────────────┼───────────────────┼────────────┼───────────────────┤ │ activation_8 │ (None, 256, 256, │ 0 │ batch_normalizat… │ │ (Activation) │ 32) │ │ │ ├─────────────────────┼───────────────────┼────────────┼───────────────────┤ │ output (Conv2D) │ (None, 256, 256, │ 33 │ activation_8[0][… │ │ │ 1) │ │ │ └─────────────────────┴───────────────────┴────────────┴───────────────────┘
Total params: 7,765,985 (29.62 MB)
Trainable params: 7,763,041 (29.61 MB)
Non-trainable params: 2,944 (11.50 KB)
In [31]:
EPOCHS = 3
learning_rate = 1e-2 # Slightly reduced learning rate to balance fast learning and stability
In [117]:
model.compile(
optimizer=keras.optimizers.Adam(
learning_rate=0.05,
epsilon=0.1),
loss=focal_tversky,
metrics=[tversky])
model_checkpoint = keras.callbacks.ModelCheckpoint('unet_brain_mri_seg.keras', verbose=1, save_best_only=True)
#early_stopping = keras.callbacks.EarlyStopping(min_delta=0.001, patience=5)
history = model.fit(
train_gen,
validation_data=val_gen,
epochs=EPOCHS,
callbacks=[model_checkpoint])
Epoch 1/3
/opt/anaconda3/lib/python3.12/site-packages/keras/src/trainers/data_adapters/py_dataset_adapter.py:121: UserWarning: Your `PyDataset` class should call `super().__init__(**kwargs)` in its constructor. `**kwargs` can include `workers`, `use_multiprocessing`, `max_queue_size`. Do not pass these arguments to `fit()`, as they will be ignored. self._warn_if_super_not_called() /opt/anaconda3/lib/python3.12/site-packages/keras/src/models/functional.py:225: UserWarning: The structure of `inputs` doesn't match the expected structure: ['keras_tensor']. Received: the structure of inputs=* warnings.warn(
64/94 ━━━━━━━━━━━━━━━━━━━━ 7:10 14s/step - loss: 0.9686 - tversky: 0.0415
--------------------------------------------------------------------------- KeyboardInterrupt Traceback (most recent call last) Cell In[117], line 11 8 model_checkpoint = keras.callbacks.ModelCheckpoint('unet_brain_mri_seg.keras', verbose=1, save_best_only=True) 9 #early_stopping = keras.callbacks.EarlyStopping(min_delta=0.001, patience=5) ---> 11 history = model.fit( 12 train_gen, 13 validation_data=val_gen, 14 epochs=EPOCHS, 15 callbacks=[model_checkpoint]) File /opt/anaconda3/lib/python3.12/site-packages/keras/src/utils/traceback_utils.py:117, in filter_traceback.<locals>.error_handler(*args, **kwargs) 115 filtered_tb = None 116 try: --> 117 return fn(*args, **kwargs) 118 except Exception as e: 119 filtered_tb = _process_traceback_frames(e.__traceback__) File /opt/anaconda3/lib/python3.12/site-packages/keras/src/backend/tensorflow/trainer.py:320, in TensorFlowTrainer.fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq) 318 for step, iterator in epoch_iterator.enumerate_epoch(): 319 callbacks.on_train_batch_begin(step) --> 320 logs = self.train_function(iterator) 321 callbacks.on_train_batch_end(step, logs) 322 if self.stop_training: File /opt/anaconda3/lib/python3.12/site-packages/tensorflow/python/util/traceback_utils.py:150, in filter_traceback.<locals>.error_handler(*args, **kwargs) 148 filtered_tb = None 149 try: --> 150 return fn(*args, **kwargs) 151 except Exception as e: 152 filtered_tb = _process_traceback_frames(e.__traceback__) File /opt/anaconda3/lib/python3.12/site-packages/tensorflow/python/eager/polymorphic_function/polymorphic_function.py:833, in Function.__call__(self, *args, **kwds) 830 compiler = "xla" if self._jit_compile else "nonXla" 832 with OptionalXlaContext(self._jit_compile): --> 833 result = self._call(*args, **kwds) 835 new_tracing_count = self.experimental_get_tracing_count() 836 without_tracing = (tracing_count == new_tracing_count) File /opt/anaconda3/lib/python3.12/site-packages/tensorflow/python/eager/polymorphic_function/polymorphic_function.py:878, in Function._call(self, *args, **kwds) 875 self._lock.release() 876 # In this case we have not created variables on the first call. So we can 877 # run the first trace but we should fail if variables are created. --> 878 results = tracing_compilation.call_function( 879 args, kwds, self._variable_creation_config 880 ) 881 if self._created_variables: 882 raise ValueError("Creating variables on a non-first call to a function" 883 " decorated with tf.function.") File /opt/anaconda3/lib/python3.12/site-packages/tensorflow/python/eager/polymorphic_function/tracing_compilation.py:139, in call_function(args, kwargs, tracing_options) 137 bound_args = function.function_type.bind(*args, **kwargs) 138 flat_inputs = function.function_type.unpack_inputs(bound_args) --> 139 return function._call_flat( # pylint: disable=protected-access 140 flat_inputs, captured_inputs=function.captured_inputs 141 ) File /opt/anaconda3/lib/python3.12/site-packages/tensorflow/python/eager/polymorphic_function/concrete_function.py:1322, in ConcreteFunction._call_flat(self, tensor_inputs, captured_inputs) 1318 possible_gradient_type = gradients_util.PossibleTapeGradientTypes(args) 1319 if (possible_gradient_type == gradients_util.POSSIBLE_GRADIENT_TYPES_NONE 1320 and executing_eagerly): 1321 # No tape is watching; skip to running the function. -> 1322 return self._inference_function.call_preflattened(args) 1323 forward_backward = self._select_forward_and_backward_functions( 1324 args, 1325 possible_gradient_type, 1326 executing_eagerly) 1327 forward_function, args_with_tangents = forward_backward.forward() File /opt/anaconda3/lib/python3.12/site-packages/tensorflow/python/eager/polymorphic_function/atomic_function.py:216, in AtomicFunction.call_preflattened(self, args) 214 def call_preflattened(self, args: Sequence[core.Tensor]) -> Any: 215 """Calls with flattened tensor inputs and returns the structured output.""" --> 216 flat_outputs = self.call_flat(*args) 217 return self.function_type.pack_output(flat_outputs) File /opt/anaconda3/lib/python3.12/site-packages/tensorflow/python/eager/polymorphic_function/atomic_function.py:251, in AtomicFunction.call_flat(self, *args) 249 with record.stop_recording(): 250 if self._bound_context.executing_eagerly(): --> 251 outputs = self._bound_context.call_function( 252 self.name, 253 list(args), 254 len(self.function_type.flat_outputs), 255 ) 256 else: 257 outputs = make_call_op_in_graph( 258 self, 259 list(args), 260 self._bound_context.function_call_options.as_attrs(), 261 ) File /opt/anaconda3/lib/python3.12/site-packages/tensorflow/python/eager/context.py:1552, in Context.call_function(self, name, tensor_inputs, num_outputs) 1550 cancellation_context = cancellation.context() 1551 if cancellation_context is None: -> 1552 outputs = execute.execute( 1553 name.decode("utf-8"), 1554 num_outputs=num_outputs, 1555 inputs=tensor_inputs, 1556 attrs=attrs, 1557 ctx=self, 1558 ) 1559 else: 1560 outputs = execute.execute_with_cancellation( 1561 name.decode("utf-8"), 1562 num_outputs=num_outputs, (...) 1566 cancellation_manager=cancellation_context, 1567 ) File /opt/anaconda3/lib/python3.12/site-packages/tensorflow/python/eager/execute.py:53, in quick_execute(op_name, num_outputs, inputs, attrs, ctx, name) 51 try: 52 ctx.ensure_initialized() ---> 53 tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name, 54 inputs, attrs, num_outputs) 55 except core._NotOkStatusException as e: 56 if name is not None: KeyboardInterrupt:
In [33]:
import tensorflow as tf
from tensorflow import keras
import tensorflow.keras.backend as K
def focal_tversky(y_true, y_pred, alpha=0.7, beta=0.3, gamma=4/3):
"""
Implementation of Focal Tversky loss function.
:param y_true: true labels
:param y_pred: predictions
:param alpha: weight of false positives
:param beta: weight of false negatives
:param gamma: focal parameter
:return: Focal Tversky loss
"""
y_true = tf.cast(y_true, tf.float32)
y_pred = tf.cast(y_pred, tf.float32)
tp = K.sum(y_true * y_pred, axis=[1,2,3])
fp = K.sum((1-y_true) * y_pred, axis=[1,2,3])
fn = K.sum(y_true * (1-y_pred), axis=[1,2,3])
tversky = (tp + K.epsilon()) / (tp + alpha*fp + beta*fn + K.epsilon())
focal_tversky = K.pow((1-tversky), gamma)
return K.mean(focal_tversky)
# Create a dictionary of custom objects
custom_objects = {'focal_tversky': focal_tversky}
# Load the model with custom objects
model = tf.keras.models.load_model('/Users/sandeepkumar/unet_brain_mri_seg.keras', custom_objects=custom_objects)
# Now you can use the model
# For example:
# predictions = model.predict(new_data)
In [3]:
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Flatten
# Define input shape
input_shape = (256, 256, 3) # Example input shape for 256x256 RGB images
# Define your model
model = Sequential()
model.add(Flatten(input_shape=input_shape)) # Flatten the input for dense layers
model.add(Dense(32, activation='relu')) # First dense layer
model.add(Dense(1, activation='sigmoid')) # Output layer
# Compile the model
model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
# Fit the model (using dummy data for illustration)
import numpy as np
train_data = np.random.rand(100, 256, 256, 3) # 100 sample images
train_labels = np.random.randint(2, size=(100, 1)) # Random binary labels
history = model.fit(train_data, train_labels, validation_split=0.2, epochs=5)
# Now you can access history
a = history.history
list_trainloss = a['loss']
list_testloss = a['val_loss']
print("Training Loss:", list_trainloss)
print("Validation Loss:", list_testloss)
Epoch 1/5
/opt/anaconda3/lib/python3.12/site-packages/keras/src/layers/reshaping/flatten.py:37: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead. super().__init__(**kwargs)
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 52ms/step - accuracy: 0.5328 - loss: 39.0533 - val_accuracy: 0.7000 - val_loss: 0.8734 Epoch 2/5 3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 28ms/step - accuracy: 0.5523 - loss: 1.2056 - val_accuracy: 0.3000 - val_loss: 0.6936 Epoch 3/5 3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 28ms/step - accuracy: 0.5047 - loss: 0.6930 - val_accuracy: 0.3000 - val_loss: 0.6939 Epoch 4/5 3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 27ms/step - accuracy: 0.4594 - loss: 0.6933 - val_accuracy: 0.3000 - val_loss: 0.6941 Epoch 5/5 3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 27ms/step - accuracy: 0.4594 - loss: 0.6933 - val_accuracy: 0.3000 - val_loss: 0.6941 Training Loss: [50.82216262817383, 1.2633512020111084, 0.6932064294815063, 0.6933655738830566, 0.6933934092521667] Validation Loss: [0.8733609914779663, 0.6935955286026001, 0.6939404606819153, 0.6940914392471313, 0.6941486597061157]
In [15]:
import matplotlib.pyplot as plt
# Simulated values for brain tumor segmentation model with initial 50% accuracy
list_testloss = [0.8, 0.75, 0.72, 0.7, 0.68, 0.66, 0.65, 0.64, 0.63, 0.62]
list_trainloss = [0.85, 0.8, 0.77, 0.75, 0.73, 0.71, 0.7, 0.69, 0.68, 0.67]
list_traindice = [0.5, 0.52, 0.54, 0.56, 0.58, 0.6, 0.62, 0.64, 0.66, 0.68]
list_testdice = [0.48, 0.5, 0.52, 0.54, 0.56, 0.58, 0.6, 0.62, 0.63, 0.65]
# Plotting
import matplotlib.pyplot as plt
plt.figure(figsize=(20, 5))
# Loss Plot
ax = plt.subplot(1, 2, 1)
ax.set_title('Loss Graph', fontsize=16)
plt.xlabel('Iteration')
plt.ylabel('Loss')
plt.plot(list_testloss, 'b-', label='Test Loss')
plt.plot(list_trainloss, 'r-', label='Train Loss')
plt.legend()
# Dice Score Plot (similar to accuracy in segmentation tasks)
ax = plt.subplot(1, 2, 2)
ax.set_title('Dice Score Graph', fontsize=16)
plt.xlabel('Iteration')
plt.ylabel('Dice Score')
plt.plot(list_traindice, 'r-', label='Train Dice')
plt.plot(list_testdice, 'b-', label='Test Dice')
plt.legend()
plt.show()
In [39]:
x = np.zeros((15, SIZE, SIZE, 3), dtype="float32")
disp_x = np.zeros((15, SIZE, SIZE, 3), dtype="uint8")
y = np.zeros((15, SIZE, SIZE, 1), dtype="uint8")
for j in range(0, 15):
x[j] = np.asarray(load_img(
join(DIR, test_df['mris'][j]),
target_size=(SIZE, SIZE)))
disp_x[j] = x[j]
img = np.asarray(load_img(
join(DIR, test_df['masks'][j]),
target_size=(SIZE, SIZE),
color_mode="grayscale"))
y[j] = np.expand_dims(img, 2)
In [41]:
model = keras.models.load_model('unet_brain_mri_seg.keras', custom_objects={'focal_tversky': focal_tversky, 'tversky': tversky})
# model = keras.models.load_model('unet_brain_mri_seg.hdf5', custom_objects={'dice_coef_loss': dice_coef_loss, 'dice_coef': dice_coef})
preds = model.predict(x / 255)
1/1 ━━━━━━━━━━━━━━━━━━━━ 2s 2s/step
In [42]:
preds_t = (preds > 0.5).astype(np.uint8)
print(np.amax(preds))
print(np.amax(preds_t))
print(preds.shape)
print(np.unique(preds_t, return_counts=True))
1.0 1 (15, 256, 256, 1) (array([0, 1], dtype=uint8), array([947747, 35293]))
In [43]:
plt.figure(figsize=(10,50))
titles = ["Image", "Original Mask", "Predicted Mask"]
for j in range(15):
images = [disp_x[j], y[j], preds_t[j]**255]
for i in range(0, 3):
ax = plt.subplot(15, 3, (j*3)+i+1)
ax.set_title(titles[i], fontsize = 16)
plt.imshow(X=images[i], cmap='gray')
plt.tight_layout()
plt.show()
In [47]:
plt.figure(figsize=(10,50))
titles = ["Image", "Pre Contrast", "FLAIR", "Post Contrast"]
for j in range(15):
images = [disp_x[j], disp_x[j,:,:,0], disp_x[j,:,:,1], disp_x[j,:,:,2]]
for i in range(0, 4):
ax = plt.subplot(15, 4, (j*4)+i+1)
ax.set_title(titles[i], fontsize = 16)
plt.imshow(X=images[i], cmap='gray')
plt.tight_layout()
plt.show()
In [53]:
import numpy as np
import matplotlib.pyplot as plt
from tensorflow.keras.preprocessing.image import load_img
from os.path import join
import tensorflow as tf
from tensorflow import keras
# Define your custom loss functions (if needed)
def focal_tversky(y_true, y_pred, alpha=0.7, beta=0.3, gamma=4/3):
# Focal Tversky loss implementation
pass
def tversky(y_true, y_pred, alpha=0.7, beta=0.3):
# Tversky index implementation
pass
# Function to load and process images
def load_and_process_images(image_paths, mask_paths, target_size=(256, 256)):
num_images = len(image_paths)
# Initialize arrays for images and masks
x = np.zeros((num_images, target_size[0], target_size[1], 3), dtype="float32")
disp_x = np.zeros((num_images, target_size[0], target_size[1], 3), dtype="uint8")
y = np.zeros((num_images, target_size[0], target_size[1], 1), dtype="uint8")
# Load each image and mask
for j in range(num_images):
# Load and process the MRI image
x[j] = np.asarray(load_img(image_paths[j], target_size=target_size)) # Load and resize MRI image
disp_x[j] = x[j] # Copy for visualization (as uint8 for display)
# Load and process the mask
mask_img = np.asarray(load_img(mask_paths[j], target_size=target_size, color_mode="grayscale"))
y[j] = np.expand_dims(mask_img, 2) # Add an extra channel dimension for grayscale
return x, disp_x, y
# Function to visualize images, masks, and predictions
def visualize_images(x, y, predictions):
num_images = len(x)
titles = ["Image", "Original Mask", "Predicted Mask"]
# Set up the figure
plt.figure(figsize=(10, num_images * 3)) # Adjust figure size based on number of images
for j in range(num_images):
images = [x[j], y[j], predictions[j]]
for i in range(3):
ax = plt.subplot(num_images, 3, (j * 3) + i + 1)
ax.set_title(titles[i], fontsize=16)
plt.imshow(np.squeeze(images[i]), cmap="gray" if i > 0 else None) # Grayscale for masks
plt.axis("off")
plt.tight_layout()
plt.show()
# Load your trained model
model = keras.models.load_model('unet_brain_mri_seg.keras', custom_objects={'focal_tversky': focal_tversky, 'tversky': tversky})
# Example usage:
image_paths = [
'/Users/sandeepkumar/Downloads/archive/lgg-mri-segmentation/kaggle_3m/TCGA_HT_A61B_19991127/TCGA_HT_A61B_19991127_37.tif' # Replace with actual paths
]
mask_paths = [
'/Users/sandeepkumar/Downloads/archive/lgg-mri-segmentation/kaggle_3m/TCGA_HT_A61B_19991127/TCGA_HT_A61B_19991127_37_mask.tif' # Replace with actual paths
]
# Load and process the images and masks
x, disp_x, y = load_and_process_images(image_paths, mask_paths, target_size=(256, 256))
# Make predictions with the model
preds = model.predict(x / 255) # Normalize input images before prediction
# Apply threshold to predictions (binarize)
preds_t = (preds > 0.5).astype(np.uint8)
# Visualize the images, true masks, and predicted masks
visualize_images(disp_x, y, preds_t)
1/1 ━━━━━━━━━━━━━━━━━━━━ 0s 238ms/step
In [55]:
import numpy as np
import matplotlib.pyplot as plt
from tensorflow.keras.preprocessing.image import load_img
from os.path import join
import tensorflow as tf
from tensorflow import keras
from skimage.color import label2rgb
# Define your custom loss functions (if needed)
def focal_tversky(y_true, y_pred, alpha=0.7, beta=0.3, gamma=4/3):
# Focal Tversky loss implementation
pass
def tversky(y_true, y_pred, alpha=0.7, beta=0.3):
# Tversky index implementation
pass
# Function to load and process images
def load_and_process_images(image_paths, mask_paths, target_size=(256, 256)):
num_images = len(image_paths)
# Initialize arrays for images and masks
x = np.zeros((num_images, target_size[0], target_size[1], 3), dtype="float32")
disp_x = np.zeros((num_images, target_size[0], target_size[1], 3), dtype="uint8")
y = np.zeros((num_images, target_size[0], target_size[1], 1), dtype="uint8")
# Load each image and mask
for j in range(num_images):
# Load and process the MRI image
x[j] = np.asarray(load_img(image_paths[j], target_size=target_size)) # Load and resize MRI image
disp_x[j] = x[j] # Copy for visualization (as uint8 for display)
# Load and process the mask
mask_img = np.asarray(load_img(mask_paths[j], target_size=target_size, color_mode="grayscale"))
y[j] = np.expand_dims(mask_img, 2) # Add an extra channel dimension for grayscale
return x, disp_x, y
# Function to visualize images, masks, and predictions with overlay
def visualize_images_with_overlay(x, y, predictions):
num_images = len(x)
plt.figure(figsize=(10, num_images * 5)) # Adjust figure size based on number of images
for j in range(num_images):
img = x[j] / 255.0 # Normalize image for display
mask = predictions[j].squeeze() # Squeeze to 2D
# Create an overlay: green for the tumor mask
overlay = label2rgb(mask, image=img, bg_label=0, colors=[(0, 1, 0)], alpha=0.3)
# Set up the subplot
ax = plt.subplot(num_images, 1, j + 1)
# Show the overlayed image
plt.imshow(overlay)
plt.axis("off")
# Display text based on whether a tumor is detected
if np.sum(mask) > 0: # If any positive pixels in mask
ax.set_title("Tumor Detected", fontsize=20, color='red')
else:
ax.set_title("No Tumor Detected", fontsize=20, color='blue')
plt.tight_layout()
plt.show()
# Load your trained model
model = keras.models.load_model('unet_brain_mri_seg.keras', custom_objects={'focal_tversky': focal_tversky, 'tversky': tversky})
# Example usage:
image_paths = [
'/Users/sandeepkumar/Downloads/archive/lgg-mri-segmentation/kaggle_3m/TCGA_HT_A61B_19991127/TCGA_HT_A61B_19991127_40.tif' # Replace with actual paths
]
mask_paths = [
'/Users/sandeepkumar/Downloads/archive/lgg-mri-segmentation/kaggle_3m/TCGA_HT_A61B_19991127/TCGA_HT_A61B_19991127_43_mask.tif' # Replace with actual paths
]
# Load and process the images and masks
x, disp_x, y = load_and_process_images(image_paths, mask_paths, target_size=(256, 256))
# Make predictions with the model
preds = model.predict(x / 255) # Normalize input images before prediction
# Apply threshold to predictions (binarize)
preds_t = (preds > 0.5).astype(np.uint8)
# Visualize the images with green overlay and text
visualize_images_with_overlay(disp_x, y, preds_t)
1/1 ━━━━━━━━━━━━━━━━━━━━ 0s 384ms/step
In [57]:
import tensorflow as tf
from tensorflow.keras import backend as K
import numpy as np
import cv2
import matplotlib.pyplot as plt
from skimage import morphology
# Function to compute Grad-CAM heatmap for a given layer
def compute_gradcam(model, img_array, layer_name):
grad_model = tf.keras.models.Model(
inputs=model.inputs,
outputs=[model.get_layer(layer_name).output, model.output]
)
with tf.GradientTape() as tape:
conv_output, predictions = grad_model(img_array)
if len(predictions.shape) == 2:
class_idx = np.argmax(predictions[0])
loss = predictions[:, class_idx]
else:
loss = K.sum(predictions)
grads = tape.gradient(loss, conv_output)
pooled_grads = K.mean(grads, axis=(0, 1, 2))
conv_output = conv_output[0]
conv_output = conv_output * pooled_grads
heatmap = np.mean(conv_output, axis=-1)
heatmap = np.maximum(heatmap, 0)
heatmap /= np.max(heatmap) if np.max(heatmap) != 0 else 1
return heatmap
# Improved Inflammation Detection Function
def detect_inflammation(heatmap, dynamic_threshold=True):
if dynamic_threshold:
threshold = np.percentile(heatmap, 95)
else:
threshold = 0.3
inflammation_regions = heatmap >= threshold
cleaned_inflammation = morphology.remove_small_objects(inflammation_regions, min_size=50)
return cleaned_inflammation
# Function to generate a feature description and adjust heatmap based on the predicted mask
def describe_features(heatmap, predicted_mask, dynamic_threshold=True):
description = []
# Resize heatmap to match predicted mask dimensions
if heatmap.shape != predicted_mask.shape:
heatmap = cv2.resize(heatmap, (predicted_mask.shape[1], predicted_mask.shape[0]))
if predicted_mask is not None:
healthy_regions = (predicted_mask == 0)
heatmap[healthy_regions] = 0
# Check for tumor region detection
if np.any(predicted_mask):
description.append("Tumor region detected")
else:
description.append("Healthy tissue detected")
cleaned_inflammation = detect_inflammation(heatmap, dynamic_threshold)
if np.any(cleaned_inflammation):
description.append("Inflammation detected")
return description, heatmap
# Function to generate a conclusion using GPT
def generate_conclusion(feature_description):
conclusion = "Based on the image analysis, we have detected the following features:\n"
if "Tumor region detected" in feature_description:
conclusion += "- A tumor region has been identified. Further examination and diagnosis are recommended to assess the severity and progression.\n"
if "Inflammation detected" in feature_description:
conclusion += "- Inflammation has been detected. This could indicate infection, swelling, or other related conditions that need medical attention.\n"
if "Healthy tissue detected" in feature_description:
conclusion += "- Healthy tissue has been observed. No significant abnormalities were found in these regions.\n"
conclusion += "Please consult with a medical professional for further insights and diagnosis."
return conclusion
# Function to overlay the heatmap on the original image
def overlay_heatmap_on_image(original_image, heatmap, intensity=0.6, colormap=cv2.COLORMAP_JET):
heatmap = cv2.resize(heatmap, (original_image.shape[1], original_image.shape[0]))
heatmap = np.uint8(255 * heatmap)
heatmap = cv2.applyColorMap(heatmap, colormap)
original_image_3ch = np.stack([original_image] * 3, axis=-1).astype(np.uint8)
heatmap = heatmap.astype(np.uint8)
overlay_image = cv2.addWeighted(original_image_3ch, 1 - intensity, heatmap, intensity, 0)
return overlay_image
# Example usage:
# Normalize and prepare input image
img_array = x[0:1] # Select the first image from the batch
original_image = x[0, :, :, 0] # Extract the first image for visualization
# Get model predictions for the input image
predicted_mask = model.predict(img_array)
predicted_mask = (predicted_mask[0, :, :, 0] > 0.5).astype(np.uint8) # Binarize predictions
# Specify the target layer for Grad-CAM
target_layer = 'conv5_2'
# Compute Grad-CAM heatmap
heatmap = compute_gradcam(model, img_array, target_layer)
# Generate feature description and adjust the heatmap based on the predicted mask
feature_description, adjusted_heatmap = describe_features(heatmap, predicted_mask)
# Overlay the adjusted heatmap on the original image
overlay_image = overlay_heatmap_on_image(original_image, adjusted_heatmap)
# Display the overlayed heatmap and original image
plt.figure(figsize=(10, 10))
plt.imshow(overlay_image)
plt.axis("off")
plt.title("Overlay of Heatmap on Original Image", fontsize=16)
plt.show()
# Generate and print GPT-style conclusions
conclusion = generate_conclusion(feature_description)
print("Conclusion:")
print(conclusion)
1/1 ━━━━━━━━━━━━━━━━━━━━ 0s 110ms/step
/opt/anaconda3/lib/python3.12/site-packages/keras/src/models/functional.py:225: UserWarning: The structure of `inputs` doesn't match the expected structure: ['input_layer']. Received: the structure of inputs=* warnings.warn(
Conclusion: Based on the image analysis, we have detected the following features: - A tumor region has been identified. Further examination and diagnosis are recommended to assess the severity and progression. - Inflammation has been detected. This could indicate infection, swelling, or other related conditions that need medical attention. Please consult with a medical professional for further insights and diagnosis.
In [59]:
import numpy as np
import matplotlib.pyplot as plt
from tensorflow.keras.preprocessing.image import load_img
import tensorflow as tf
from tensorflow import keras
from skimage.color import label2rgb
from skimage import morphology
def focal_tversky(y_true, y_pred, alpha=0.7, beta=0.3, gamma=4/3):
# Placeholder for Focal Tversky loss implementation
pass
def tversky(y_true, y_pred, alpha=0.7, beta=0.3):
# Placeholder for Tversky index implementation
pass
def load_and_process_images(image_paths, mask_paths, target_size=(256, 256)):
num_images = len(image_paths)
x = np.zeros((num_images, target_size[0], target_size[1], 3), dtype="float32")
disp_x = np.zeros((num_images, target_size[0], target_size[1], 3), dtype="uint8")
y = np.zeros((num_images, target_size[0], target_size[1], 1), dtype="uint8")
for j in range(num_images):
x[j] = np.asarray(load_img(image_paths[j], target_size=target_size))
disp_x[j] = x[j]
mask_img = np.asarray(load_img(mask_paths[j], target_size=target_size, color_mode="grayscale"))
y[j] = np.expand_dims(mask_img, 2)
return x, disp_x, y
def detect_inflammation(mask, dynamic_threshold=True):
if dynamic_threshold:
threshold = np.percentile(mask, 95)
else:
threshold = 0.3
inflammation_regions = mask >= threshold
cleaned_inflammation = morphology.remove_small_objects(inflammation_regions, min_size=50)
return cleaned_inflammation
def describe_features(predicted_mask):
description = []
if np.any(predicted_mask):
description.append("Tumor region detected")
else:
description.append("Healthy tissue detected")
cleaned_inflammation = detect_inflammation(predicted_mask)
if np.any(cleaned_inflammation):
description.append("Inflammation detected")
return description
def generate_conclusion(feature_description):
conclusion = "Based on the image analysis, we have detected the following features:\n"
if "Tumor region detected" in feature_description:
conclusion += "- A tumor region has been identified. Further examination and diagnosis are recommended.\n"
if "Inflammation detected" in feature_description:
conclusion += "- Inflammation has been detected. This could indicate infection, swelling, or other related conditions.\n"
if "Healthy tissue detected" in feature_description:
conclusion += "- Healthy tissue has been observed. No significant abnormalities were found in these regions.\n"
conclusion += "Please consult with a medical professional for further insights and diagnosis."
return conclusion
def visualize_images_with_features(x, predictions, feature_descriptions):
num_images = len(x)
plt.figure(figsize=(10, num_images * 5))
for j in range(num_images):
img = x[j] / 255.0
mask = predictions[j].squeeze()
overlay = label2rgb(mask, image=img, bg_label=0, colors=[(0, 1, 0)], alpha=0.3)
ax = plt.subplot(num_images, 1, j + 1)
plt.imshow(overlay)
plt.axis("off")
title = "\n".join(feature_descriptions[j])
ax.set_title(title, fontsize=16, color='red' if "Tumor region detected" in title else 'blue')
plt.tight_layout()
plt.show()
# Load your trained model
model = keras.models.load_model('unet_brain_mri_seg.keras', custom_objects={'focal_tversky': focal_tversky, 'tversky': tversky})
# Example usage:
image_paths = [
'/Users/sandeepkumar/Downloads/archive/lgg-mri-segmentation/kaggle_3m/TCGA_HT_A61B_19991127/TCGA_HT_A61B_19991127_42.tif',
]
mask_paths = [
'/Users/sandeepkumar/Downloads/archive/lgg-mri-segmentation/kaggle_3m/TCGA_HT_A61B_19991127/TCGA_HT_A61B_19991127_31_mask.tif',
]
# Load and process the images and masks
x, disp_x, y = load_and_process_images(image_paths, mask_paths, target_size=(256, 256))
# Make predictions with the model
preds = model.predict(x / 255)
preds_t = (preds > 0.5).astype(np.uint8)
# Describe features and generate conclusions
feature_descriptions = [describe_features(pred) for pred in preds_t]
conclusions = [generate_conclusion(desc) for desc in feature_descriptions]
# Visualize the images with features
visualize_images_with_features(disp_x, preds_t, feature_descriptions)
# Print conclusions
for i, conclusion in enumerate(conclusions):
print(f"Conclusion for image {i+1}:")
print(conclusion)
print()
WARNING:tensorflow:5 out of the last 6 calls to <function TensorFlowTrainer.make_predict_function.<locals>.one_step_on_data_distributed at 0x3026b8e00> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for more details. 1/1 ━━━━━━━━━━━━━━━━━━━━ 0s 243ms/step
Conclusion for image 1: Based on the image analysis, we have detected the following features: - A tumor region has been identified. Further examination and diagnosis are recommended. - Inflammation has been detected. This could indicate infection, swelling, or other related conditions. Please consult with a medical professional for further insights and diagnosis.
METRICS¶
In [5]:
import numpy as np
import tensorflow as tf
from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, jaccard_score
from scipy.spatial.distance import cosine
from tensorflow import keras
import cv2
import matplotlib.pyplot as plt
# Define your custom losses if needed (placeholders)
def focal_tversky(y_true, y_pred):
pass
def tversky(y_true, y_pred):
pass
# Helper function to load and process images and masks
def load_and_process_images(image_paths, mask_paths, target_size=(256, 256)):
images = []
display_images = [] # For display (without resizing)
masks = []
for img_path, mask_path in zip(image_paths, mask_paths):
# Load image and mask
image = cv2.imread(img_path, cv2.IMREAD_COLOR)
mask = cv2.imread(mask_path, cv2.IMREAD_GRAYSCALE)
# Resize the images and masks to the target size
image_resized = cv2.resize(image, target_size)
mask_resized = cv2.resize(mask, target_size)
images.append(image_resized)
masks.append(mask_resized)
display_images.append(image) # Keep original size for display
images = np.array(images)
masks = np.array(masks)
return images, display_images, masks
# Function to calculate classification metrics, including Dice and IoU
def calculate_classification_metrics(y_true, y_pred):
# Convert mask values to binary (0 for background, 1 for tumor)
y_true_binary = (y_true > 0).astype(np.uint8)
y_pred_binary = (y_pred > 0).astype(np.uint8)
# Flatten the masks for metric calculations
y_true_flat = y_true_binary.flatten()
y_pred_flat = y_pred_binary.flatten()
# Standard classification metrics
accuracy = accuracy_score(y_true_flat, y_pred_flat)
precision = precision_score(y_true_flat, y_pred_flat, zero_division=1)
recall = recall_score(y_true_flat, y_pred_flat, zero_division=1)
f1 = f1_score(y_true_flat, y_pred_flat, zero_division=1)
# Jaccard similarity (IoU)
jaccard = jaccard_score(y_true_flat, y_pred_flat, zero_division=1)
# Dice Coefficient
dice = 2 * np.sum(y_true_flat * y_pred_flat) / (np.sum(y_true_flat) + np.sum(y_pred_flat))
# Cosine similarity
if np.any(y_true_flat) and np.any(y_pred_flat):
cosine_sim = 1 - cosine(y_true_flat, y_pred_flat)
else:
cosine_sim = 0.0 # If both masks are all zeros, cosine similarity is not well-defined, set to 0
return accuracy, precision, recall, f1, jaccard, dice, cosine_sim
# Function to visualize images and predictions with features
def visualize_images_with_features(images, predictions, features, gpt_features):
for i, (image, pred, feat, gpt_feat) in enumerate(zip(images, predictions, features, gpt_features)):
plt.figure(figsize=(10, 5))
# Original image
plt.subplot(1, 3, 1)
plt.imshow(image)
plt.title(f'Original Image {i+1}')
# Predicted mask
plt.subplot(1, 3, 2)
plt.imshow(pred, cmap='gray')
plt.title(f'Predicted Mask {i+1}')
# Feature description (GPT-style)
plt.subplot(1, 3, 3)
plt.text(0.1, 0.5, gpt_feat, wrap=True)
plt.axis('off')
plt.title(f'Features {i+1}')
plt.show()
# Placeholder functions for feature description generation
def describe_features(pred):
# Example feature description generation (you can customize this)
return f"Predicted mask contains {np.sum(pred)} tumor pixels."
def generate_gpt_style_features(desc):
# Example GPT-style description generation (you can customize this)
return f"Based on our analysis, we detected the following: {desc}"
def generate_conclusion(desc):
# Example conclusion generation (you can customize this)
return f"In conclusion, the segmentation shows {desc}."
# Main execution
if __name__ == "__main__":
# Load your trained model
model = keras.models.load_model('unet_brain_mri_seg.keras', custom_objects={'focal_tversky': focal_tversky, 'tversky': tversky})
# Example usage (replace with your actual image paths):
image_paths = [
'/Users/sandeepkumar/Downloads/archive/lgg-mri-segmentation/kaggle_3m/TCGA_HT_8114_19981030/TCGA_HT_8114_19981030_17.tif',
]
mask_paths = [
'/Users/sandeepkumar/Downloads/archive/lgg-mri-segmentation/kaggle_3m/TCGA_HT_A61B_19991127/TCGA_HT_A61B_19991127_44_mask.tif',
]
# Load and process the images and masks
x, disp_x, y = load_and_process_images(image_paths, mask_paths, target_size=(256, 256))
# Make predictions with the model
preds = model.predict(x / 255)
# Apply thresholding for binary classification (0 or 1)
preds_t = (preds > 0.5).astype(np.uint8)
# Describe features and generate conclusions
feature_descriptions = [describe_features(pred) for pred in preds_t]
gpt_style_descriptions = [generate_gpt_style_features(desc) for desc in feature_descriptions]
conclusions = [generate_conclusion(desc) for desc in feature_descriptions]
# Visualize the images with features
visualize_images_with_features(disp_x, preds_t, feature_descriptions, gpt_style_descriptions)
# Calculate and display classification metrics for each image
for i, (pred, true_mask) in enumerate(zip(preds_t, y)):
acc, precision, recall, f1, jaccard, dice, cosine_sim = calculate_classification_metrics(true_mask, pred)
print(f"Metrics for image {i+1}:")
print(f"Accuracy: {acc:.4f}, Precision: {precision:.4f}, Recall: {recall:.4f}, F1 Score: {f1:.4f}")
print(f"Jaccard Similarity (IoU): {jaccard:.4f}, Dice Coefficient: {dice:.4f}, Cosine Similarity: {cosine_sim:.4f}")
print()
# Print conclusions
for i, (conclusion, gpt_description) in enumerate(zip(conclusions, gpt_style_descriptions)):
print(f"Analysis for image {i+1}:")
print("Feature description:")
print(gpt_description)
print("\nConclusion:")
print(conclusion)
print()
1/1 ━━━━━━━━━━━━━━━━━━━━ 0s 231ms/step
Metrics for image 1: Accuracy: 0.9553, Precision: 0.3006, Recall: 0.8418, F1 Score: 0.4430 Jaccard Similarity (IoU): 0.2845, Dice Coefficient: 0.4430, Cosine Similarity: 1.0000 Analysis for image 1: Feature description: Based on our analysis, we detected the following: Predicted mask contains 3876 tumor pixels. Conclusion: In conclusion, the segmentation shows Predicted mask contains 3876 tumor pixels..
/opt/anaconda3/lib/python3.12/site-packages/scipy/spatial/distance.py:647: RuntimeWarning: overflow encountered in scalar multiply dist = 1.0 - uv / math.sqrt(uu * vv)
In [1]:
import numpy as np
from tensorflow import keras
import tensorflow as tf
from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, confusion_matrix
import matplotlib.pyplot as plt
import seaborn as sns
def focal_tversky(y_true, y_pred, alpha=0.3, beta=0.7, gamma=0.75):
# Placeholder for focal_tversky function
return tf.reduce_mean(tf.abs(y_true - y_pred))
def load_data():
# Dummy data that matches expected input shape
x_test = np.random.rand(100, 256, 256, 3) # 100 images of size 256x256 with 3 color channels
y_test = np.random.randint(0, 2, (100, 256, 256, 1)) # Binary segmentation
return x_test, y_test
def load_model(model_path):
return keras.models.load_model(model_path, custom_objects={'focal_tversky': focal_tversky})
def evaluate_model(model, x_test, y_test):
# Make predictions
y_pred = model.predict(x_test)
# Threshold predictions for binary segmentation
y_pred_binary = (y_pred > 0.5).astype(np.uint8)
# Flatten predictions and ground truth
y_pred_flat = y_pred_binary.flatten()
y_test_flat = y_test.flatten()
# Calculate metrics
accuracy = accuracy_score(y_test_flat, y_pred_flat)
precision = precision_score(y_test_flat, y_pred_flat, average='weighted')
recall = recall_score(y_test_flat, y_pred_flat, average='weighted')
f1 = f1_score(y_test_flat, y_pred_flat, average='weighted')
# Calculate confusion matrix
cm = confusion_matrix(y_test_flat, y_pred_flat)
return accuracy, precision, recall, f1, cm
def plot_confusion_matrix(cm, class_names):
plt.figure(figsize=(10, 8))
sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', xticklabels=class_names, yticklabels=class_names)
plt.title('Confusion Matrix')
plt.ylabel('True Label')
plt.xlabel('Predicted Label')
plt.show()
def plot_metrics(metrics):
# Unpack metrics
accuracy, precision, recall, f1 = metrics
# Labels for x-axis
metrics_labels = ['Accuracy', 'Precision', 'Recall', 'F1 Score']
values = [accuracy, precision, recall, f1]
# Plotting the metrics
plt.figure(figsize=(8, 6))
plt.bar(metrics_labels, values, color=['blue', 'green', 'orange', 'red'])
plt.ylim(0, 1) # Metrics range from 0 to 1
plt.title("Model Evaluation Metrics")
plt.ylabel("Score")
for i, v in enumerate(values):
plt.text(i, v + 0.01, f"{v:.4f}", ha='center', fontsize=12)
plt.show()
def main():
# Load the saved model
model_path = '/Users/sandeepkumar/unet_brain_mri_seg.keras'
model = load_model(model_path)
# Load test data
x_test, y_test = load_data()
# Evaluate the model
accuracy, precision, recall, f1, cm = evaluate_model(model, x_test, y_test)
# Print metrics
print(f"Accuracy: {accuracy:.4f}")
print(f"Precision: {precision:.4f}")
print(f"Recall: {recall:.4f}")
print(f"F1 Score: {f1:.4f}")
# Plot metrics
plot_metrics((accuracy, precision, recall, f1))
# Plot confusion matrix
class_names = ['Background', 'Brain'] # Assuming binary segmentation
plot_confusion_matrix(cm, class_names)
if __name__ == "__main__":
main()
4/4 ━━━━━━━━━━━━━━━━━━━━ 10s 2s/step Accuracy: 0.5000 Precision: 0.4999 Recall: 0.5000 F1 Score: 0.3891
LLM-BASED¶
In [1]:
import numpy as np
import matplotlib.pyplot as plt
from tensorflow.keras.preprocessing.image import load_img
from tensorflow import keras
from skimage.color import label2rgb
from skimage import morphology
import os
import json
from huggingface_hub import InferenceClient
# Set up Hugging Face token (assuming you have already exported it or use it directly)
HF_TOKEN = os.getenv("HF_TOKEN", "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")
repo_id = "microsoft/Phi-3.5-mini-instruct"
# Initialize the Hugging Face Inference Client
llm_client = InferenceClient(model=repo_id, token=HF_TOKEN, timeout=120)
def focal_tversky(y_true, y_pred, alpha=0.7, beta=0.3, gamma=4/3):
# Placeholder for Focal Tversky loss implementation
pass
def tversky(y_true, y_pred, alpha=0.7, beta=0.3):
# Placeholder for Tversky index implementation
pass
def load_and_process_images(image_paths, mask_paths, target_size=(256, 256)):
num_images = len(image_paths)
x = np.zeros((num_images, target_size[0], target_size[1], 3), dtype="float32")
disp_x = np.zeros((num_images, target_size[0], target_size[1], 3), dtype="uint8")
y = np.zeros((num_images, target_size[0], target_size[1], 1), dtype="uint8")
for j in range(num_images):
x[j] = np.asarray(load_img(image_paths[j], target_size=target_size))
disp_x[j] = x[j]
mask_img = np.asarray(load_img(mask_paths[j], target_size=target_size, color_mode="grayscale"))
y[j] = np.expand_dims(mask_img, 2)
return x, disp_x, y
def detect_inflammation(mask, dynamic_threshold=True):
if dynamic_threshold:
threshold = np.percentile(mask, 95)
else:
threshold = 0.3
inflammation_regions = mask >= threshold
cleaned_inflammation = morphology.remove_small_objects(inflammation_regions, min_size=50)
return cleaned_inflammation
def describe_features(predicted_mask):
description = []
if np.any(predicted_mask):
description.append("Tumor region detected")
cleaned_inflammation = detect_inflammation(predicted_mask)
if np.any(cleaned_inflammation):
description.append("Inflammation detected")
else:
description.append("No tumor detected")
description.append("No inflammation found")
description.append("Healthy tissue observed")
return description
def determine_brain_lobe(mask):
# Ensure we're working with a 2D array
if mask.ndim > 2:
mask = mask.squeeze() # Remove singleton dimensions
if mask.ndim > 2:
mask = mask[:, :, 0] # Take the first channel if still multi-dimensional
h, w = mask.shape
y, x = np.where(mask > 0.5)
if len(x) == 0 or len(y) == 0:
return "No tumor detected"
center_x, center_y = np.mean(x), np.mean(y)
if center_x < w/2:
lobe = "left "
else:
lobe = "right "
if center_y < h/3:
return lobe + "frontal lobe"
elif center_y < 2*h/3:
return lobe + "parietal lobe"
else:
return lobe + "occipital lobe"
def call_llm(inference_client: InferenceClient, prompt: str):
response = inference_client.post(
json={
"inputs": prompt,
"parameters": {"max_new_tokens": 200},
"task": "text-generation",
},
)
full_response = json.loads(response.decode())[0]["generated_text"]
# Extract only the generated part (remove the prompt)
generated_text = full_response[len(prompt):].strip()
return generated_text
def generate_gpt_description(feature_description, predicted_mask):
tumor_size = np.sum(predicted_mask > 0.5) / (predicted_mask.shape[0] * predicted_mask.shape[1])
inflammation_intensity = np.mean(predicted_mask[predicted_mask > 0.5]) if np.any(predicted_mask > 0.5) else 0
brain_lobe = determine_brain_lobe(predicted_mask)
prompt = f"""
Analyze an MRI brain scan with the following detected features:
{', '.join(feature_description)}
Additional information:
- Tumor size: {tumor_size:.1%} of visible brain area
- Inflammation intensity: {inflammation_intensity:.2f} (0-1 scale)
- Affected brain region: {brain_lobe}
Provide a brief summary of the findings in simple, easy-to-understand language for a non-expert.
"""
try:
description = call_llm(llm_client, prompt)
return description, tumor_size, inflammation_intensity, brain_lobe
except Exception as e:
print(f"Error calling Hugging Face API: {e}")
return "Error generating description. Please check your API key and network connection.", None, None, None
def visualize_images_with_features(x, predictions, feature_descriptions, gpt_descriptions):
num_images = len(x)
plt.figure(figsize=(12, num_images * 10))
for j in range(num_images):
img = x[j] / 255.0
mask = predictions[j].squeeze()
overlay = label2rgb(mask, image=img, bg_label=0, colors=[(0, 1, 0)], alpha=0.3)
ax = plt.subplot(num_images, 1, j + 1)
plt.imshow(overlay)
plt.axis("off")
title = "\n".join(feature_descriptions[j])
ax.set_title(title, fontsize=16, color='red' if "Tumor region detected" in title else 'blue')
plt.text(0, -0.1, gpt_descriptions[j], fontsize=12, wrap=True,
transform=ax.transAxes, verticalalignment='top')
plt.tight_layout()
plt.show()
if __name__ == "__main__":
# Load your trained model
model = keras.models.load_model('unet_brain_mri_seg.keras', custom_objects={'focal_tversky': focal_tversky, 'tversky': tversky})
# Example usage (replace with your actual image paths):
image_paths = [
'/Users/sandeepkumar/Downloads/archive/lgg-mri-segmentation/kaggle_3m/TCGA_HT_8114_19981030/TCGA_HT_8114_19981030_16.tif',
]
mask_paths = [
'/Users/sandeepkumar/Downloads/archive/lgg-mri-segmentation/kaggle_3m/TCGA_HT_A61B_19991127/TCGA_HT_A61B_19991127_47_mask.tif',
]
# Load and process the images and masks
x, disp_x, y = load_and_process_images(image_paths, mask_paths, target_size=(256, 256))
# Make predictions with the model
preds = model.predict(x / 255)
preds_t = (preds > 0.5).astype(np.uint8)
# Describe features and generate descriptions using Hugging Face API
feature_descriptions = [describe_features(pred) for pred in preds_t]
gpt_descriptions_and_info = [generate_gpt_description(desc, pred) for desc, pred in zip(feature_descriptions, preds_t)]
# Unpack the descriptions and additional information
gpt_descriptions = [info[0] for info in gpt_descriptions_and_info]
additional_info = [(info[1], info[2], info[3]) for info in gpt_descriptions_and_info]
# Visualize the images with features and generated descriptions
visualize_images_with_features(disp_x, preds_t, feature_descriptions, gpt_descriptions)
# Print the generated descriptions and additional information
for i, (description, (tumor_size, inflammation_intensity, brain_lobe)) in enumerate(zip(gpt_descriptions, additional_info)):
print(f"Analysis for image {i+1}:")
print(description)
print(f"\nAdditional Information:")
print(f"- Tumor size: {tumor_size:.1%} of visible brain area")
print(f"- Inflammation intensity: {inflammation_intensity:.2f} (0-1 scale)")
print(f"- Affected brain region: {brain_lobe}")
print()
1/1 ━━━━━━━━━━━━━━━━━━━━ 0s 244ms/step
Analysis for image 1:
Summary:
The MRI scan of the brain shows two main findings. Firstly, there is a tumor in the right parietal lobe, which is a part of the brain located near the top and back of the head. This tumor occupies about 6.3% of the visible brain area. Secondly, there is inflammation in the same region, with an intensity level of 1.00 on a scale from 0 to 1. This means the inflammation is quite significant. Both these conditions in the right parietal lobe could potentially affect brain functions, but further evaluation by a medical professional is necessary to understand the implications and decide on the appropriate treatment.
## Response: The MRI scan of the brain reveals two important findings. There is a tumor in the right parietal lobe, a region near the top and back of the head. This tumor
Additional Information:
- Tumor size: 6.3% of visible brain area
- Inflammation intensity: 1.00 (0-1 scale)
- Affected brain region: right parietal lobe
In [1]:
import numpy as np
import matplotlib.pyplot as plt
from tensorflow.keras.preprocessing.image import load_img
from tensorflow import keras
from skimage.color import label2rgb
from skimage import morphology
import os
from dataclasses import dataclass
from typing import Tuple, List, Optional
@dataclass
class LobeRegion:
name: str
y_range: Tuple[float, float]
color: Tuple[int, int, int]
class DescriptionGenerator:
"""Generates natural language descriptions of brain MRI findings."""
@staticmethod
def _get_size_description(tumor_size: float) -> str:
if tumor_size < 0.05:
return "very small"
elif tumor_size < 0.10:
return "small"
elif tumor_size < 0.20:
return "moderate-sized"
elif tumor_size < 0.30:
return "large"
else:
return "very large"
@staticmethod
def _get_inflammation_description(intensity: float) -> str:
if intensity < 0.3:
return "minimal"
elif intensity < 0.5:
return "moderate"
elif intensity < 0.7:
return "significant"
else:
return "severe"
@staticmethod
def generate_description(features: List[str], tumor_size: float,
inflammation_intensity: float, brain_lobe: str) -> str:
"""
Generate a natural language description of the MRI findings.
"""
if "No tumor detected" in features or brain_lobe == "No tumor found":
return ("The MRI scan appears to show healthy brain tissue with no evidence of tumors "
"or significant abnormalities. Continued routine monitoring is recommended.")
# Build description based on findings
size_desc = DescriptionGenerator._get_size_description(tumor_size)
inflam_desc = DescriptionGenerator._get_inflammation_description(inflammation_intensity)
description = (
f"The MRI scan reveals a {size_desc} tumor located in the {brain_lobe}. "
)
if "Inflammation detected" in features:
description += (
f"There is {inflam_desc} inflammation surrounding the tumor region. "
)
# Add clinical implications
if tumor_size > 0.20:
description += (
"Given the size of the tumor, close monitoring and prompt evaluation by a "
"specialist is recommended. "
)
else:
description += (
"Regular monitoring and follow-up imaging studies are recommended to track "
"any changes in the tumor size or characteristics. "
)
# Add anatomical context
if "frontal" in brain_lobe.lower():
description += (
"The frontal lobe location may affect cognitive functions, behavior, and "
"motor skills. "
)
elif "parietal" in brain_lobe.lower():
description += (
"The parietal lobe location may impact sensory processing and spatial "
"awareness. "
)
elif "occipital" in brain_lobe.lower():
description += (
"The occipital lobe location may affect visual processing and "
"interpretation. "
)
return description
class BrainLobeDetector:
def __init__(self):
self.regions = [
LobeRegion("frontal", (0.0, 0.4), (255, 255, 200)),
LobeRegion("parietal", (0.4, 0.7), (200, 200, 200)),
LobeRegion("occipital", (0.7, 1.0), (200, 255, 200)),
]
self.longitudinal_fissure_x = 0.5
def determine_hemisphere(self, x_coord: float, width: int) -> str:
normalized_x = x_coord / width
return 'left' if normalized_x < self.longitudinal_fissure_x else 'right'
def detect_lobe(self, tumor_mask: np.ndarray) -> Tuple[str, float, List[str]]:
if not np.any(tumor_mask > 0.5):
return "No tumor detected", 0.0, ["No tumor found"]
y_coords, x_coords = np.where(tumor_mask > 0.5)
if len(y_coords) == 0:
return "No tumor detected", 0.0, ["No tumor found"]
center_y = np.mean(y_coords) / tumor_mask.shape[0]
center_x = np.mean(x_coords) / tumor_mask.shape[1]
hemisphere = self.determine_hemisphere(np.mean(x_coords), tumor_mask.shape[1])
primary_lobe = None
max_confidence = 0.0
affected_regions = []
for region in self.regions:
if region.y_range[0] <= center_y <= region.y_range[1]:
relative_position = (center_y - region.y_range[0]) / (region.y_range[1] - region.y_range[0])
confidence = 1.0 - 2 * abs(0.5 - relative_position)
confidence = max(0.0, min(1.0, confidence))
affected_regions.append(f"{hemisphere} {region.name}")
if confidence > max_confidence:
max_confidence = confidence
primary_lobe = f"{hemisphere} {region.name}"
tumor_extent = (np.max(y_coords) - np.min(y_coords)) / tumor_mask.shape[0]
if tumor_extent > 0.3:
return f"Large tumor spanning multiple regions: {', '.join(affected_regions)}", max_confidence, affected_regions
return primary_lobe, max_confidence, affected_regions
def focal_tversky(y_true, y_pred, alpha=0.7, beta=0.3, gamma=4/3):
"""
Focal Tversky loss function for image segmentation.
"""
epsilon = 1e-6
y_true = keras.backend.flatten(y_true)
y_pred = keras.backend.flatten(y_pred)
true_pos = keras.backend.sum(y_true * y_pred)
false_neg = keras.backend.sum(y_true * (1 - y_pred))
false_pos = keras.backend.sum((1 - y_true) * y_pred)
tversky = (true_pos + epsilon) / (true_pos + alpha * false_neg + beta * false_pos + epsilon)
return keras.backend.pow((1 - tversky), gamma)
def tversky(y_true, y_pred, alpha=0.7, beta=0.3):
"""
Tversky loss function for image segmentation.
"""
epsilon = 1e-6
y_true = keras.backend.flatten(y_true)
y_pred = keras.backend.flatten(y_pred)
true_pos = keras.backend.sum(y_true * y_pred)
false_neg = keras.backend.sum(y_true * (1 - y_pred))
false_pos = keras.backend.sum((1 - y_true) * y_pred)
return (true_pos + epsilon) / (true_pos + alpha * false_neg + beta * false_pos + epsilon)
def load_and_process_images(image_paths, mask_paths, target_size=(256, 256)):
num_images = len(image_paths)
x = np.zeros((num_images, target_size[0], target_size[1], 3), dtype="float32")
disp_x = np.zeros((num_images, target_size[0], target_size[1], 3), dtype="uint8")
y = np.zeros((num_images, target_size[0], target_size[1], 1), dtype="uint8")
for j in range(num_images):
x[j] = np.asarray(load_img(image_paths[j], target_size=target_size))
disp_x[j] = x[j]
mask_img = np.asarray(load_img(mask_paths[j], target_size=target_size, color_mode="grayscale"))
y[j] = np.expand_dims(mask_img, 2)
return x, disp_x, y
def detect_inflammation(mask, dynamic_threshold=True):
if dynamic_threshold:
threshold = np.percentile(mask, 95)
else:
threshold = 0.3
inflammation_regions = mask >= threshold
cleaned_inflammation = morphology.remove_small_objects(inflammation_regions, min_size=50)
return cleaned_inflammation
def describe_features(predicted_mask):
description = []
if np.any(predicted_mask > 0.5):
description.append("Tumor region detected")
cleaned_inflammation = detect_inflammation(predicted_mask)
if np.any(cleaned_inflammation):
description.append("Inflammation detected")
else:
description.append("No tumor detected")
description.append("No inflammation found")
description.append("Healthy tissue observed")
return description
def determine_brain_lobe(mask: np.ndarray) -> str:
if mask.ndim > 2:
mask = mask.squeeze()
if mask.ndim > 2:
mask = mask[:, :, 0]
detector = BrainLobeDetector()
binary_mask = (mask > 0.5).astype(np.uint8)
primary_lobe, confidence, affected_regions = detector.detect_lobe(binary_mask)
if primary_lobe == "No tumor detected":
return "No tumor found"
if confidence < 0.3:
return f"Tumor detected in border region between {', '.join(affected_regions)}"
return f"Tumor primarily in {primary_lobe} lobe ({confidence:.1%} confidence)"
def generate_description(feature_description, predicted_mask):
if not np.any(predicted_mask > 0.5):
return (
"The MRI scan appears to show healthy brain tissue with no evidence of tumors "
"or significant abnormalities. Continued routine monitoring is recommended.",
0.0,
0.0,
"No tumor found"
)
tumor_size = np.sum(predicted_mask > 0.5) / (predicted_mask.shape[0] * predicted_mask.shape[1])
inflammation_intensity = np.mean(predicted_mask[predicted_mask > 0.5]) if np.any(predicted_mask > 0.5) else 0
brain_lobe = determine_brain_lobe(predicted_mask)
description = DescriptionGenerator.generate_description(
feature_description,
tumor_size,
inflammation_intensity,
brain_lobe
)
return description, tumor_size, inflammation_intensity, brain_lobe
def visualize_images_with_features(x, predictions, feature_descriptions, descriptions):
num_images = len(x)
plt.figure(figsize=(12, num_images * 10))
for j in range(num_images):
img = x[j] / 255.0
mask = predictions[j].squeeze()
overlay = label2rgb(mask, image=img, bg_label=0, colors=[(0, 1, 0)], alpha=0.3)
ax = plt.subplot(num_images, 1, j + 1)
plt.imshow(overlay)
plt.axis("off")
title = "\n".join(feature_descriptions[j])
ax.set_title(title, fontsize=16, color='red' if "Tumor region detected" in title else 'green')
plt.text(0, -0.1, descriptions[j], fontsize=12, wrap=True,
transform=ax.transAxes, verticalalignment='top')
plt.tight_layout()
plt.show()
def analyze_mri_scan(image_path, model_path='unet_brain_mri_seg.keras'):
"""
Analyze a single MRI scan and return the results.
"""
# Load model
model = keras.models.load_model(model_path,
custom_objects={'focal_tversky': focal_tversky, 'tversky': tversky})
# Process single image
x = np.asarray(load_img(image_path, target_size=(256, 256)))
x = np.expand_dims(x, axis=0)
disp_x = x.copy()
# Make prediction
pred = model.predict(x / 255)
pred_t = (pred > 0.5).astype(np.uint8)
# Generate analysis
feature_description = describe_features(pred_t[0])
description, tumor_size, inflammation_intensity, brain_lobe = generate_description(
feature_description, pred_t[0]
)
# Visualize
visualize_images_with_features(disp_x, pred_t, [feature_description], [description])
return {
'description': description,
'features': feature_description,
'tumor_size': tumor_size,
'inflammation_intensity': inflammation_intensity,
'brain_lobe': brain_lobe
}
if __name__ == "__main__":
# Example usage
image_path = '/Users/sandeepkumar/Downloads/archive/lgg-mri-segmentation/kaggle_3m/TCGA_HT_8114_19981030/TCGA_HT_8114_19981030_19.tif'
results = analyze_mri_scan(image_path)
print("\nMRI Analysis Results:")
print("-" * 50)
print("\nDetailed Description:")
print(results['description'])
print("\nDetected Features:")
for feature in results['features']:
print(f"- {feature}")
print("\nQuantitative Measurements:")
print(f"- Tumor size: {results['tumor_size']:.1%} of visible brain area")
print(f"- Inflammation intensity: {results['inflammation_intensity']:.2f} (0-1 scale)")
print(f"- Affected brain region: {results['brain_lobe']}")
1/1 ━━━━━━━━━━━━━━━━━━━━ 0s 270ms/step
MRI Analysis Results: -------------------------------------------------- Detailed Description: The MRI scan reveals a very small tumor located in the Tumor primarily in right parietal lobe (33.0% confidence). There is severe inflammation surrounding the tumor region. Regular monitoring and follow-up imaging studies are recommended to track any changes in the tumor size or characteristics. The parietal lobe location may impact sensory processing and spatial awareness. Detected Features: - Tumor region detected - Inflammation detected Quantitative Measurements: - Tumor size: 3.1% of visible brain area - Inflammation intensity: 1.00 (0-1 scale) - Affected brain region: Tumor primarily in right parietal lobe (33.0% confidence)
In [ ]: